package douban;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
import uk.org.lidalia.slf4jext.Logger;
import uk.org.lidalia.slf4jext.LoggerFactory;

import java.util.Calendar;

/**
 * Created by liuyaowen on 2016/8/25.
 */
public class DoubanlController {
    protected static final Logger logger = LoggerFactory.getLogger(DoubanlController.class);

    public static boolean main() throws Exception {
        String crawlStorageFolder = "douban";//项目目录
        int numberOfCrawlers = 2;//多少个线程

        CrawlConfig config = new CrawlConfig();
        config.setCrawlStorageFolder(crawlStorageFolder);

        config.setPolitenessDelay(1000);//延迟时间
        config.setMaxDepthOfCrawling(1);//最大深度

        config.setMaxPagesToFetch(1000);//设置最大页面获取

        config.setIncludeBinaryContentInCrawling(false);//设置在爬行包含二进制内容

        config.setResumableCrawling(false);//设置可恢复的爬行

        PageFetcher pageFetcher = new PageFetcher(config);
        RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
        RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
        CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

        controller.addSeed("https://www.douban.com/group/shanghaizufang/discussion?start=0");
        controller.addSeed("https://www.douban.com/group/shanghaizufang/discussion?start=25");
        controller.addSeed("https://www.douban.com/group/shanghaizufang/discussion?start=50");
        controller.addSeed("https://www.douban.com/group/shanghaizufang/discussion?start=75");
        controller.addSeed("https://www.douban.com/group/shanghaizufang/discussion?start=100");
        // controller.addSeed("https://www.douban.com/group/topic/90187680/");
        controller.start(DoubanCrawler.class, numberOfCrawlers);
        return true;
    }


    public static void main(String[] args) throws Exception {
        int i = 0;
        while (true) {
            if (main()) {
                i = i + 1;
                logger.info("第{" + i + "}次重复......Waiting for 10 seconds");
                Calendar c = Calendar.getInstance();
                int hour = c.get(Calendar.HOUR_OF_DAY);
                if (hour==23){
                    Thread.sleep(1000 * 60 * 60 * 6);//睡10分钟
                    logger.info("go to bed....");
                }else{
                    logger.info("Thread.sleep ten Minute ....");
                    Thread.sleep(1000 * 60 * 10);//睡10分钟
                }
            }
        }
    }
}
